# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from lxml import etree
from scrapy import signals, Request
from scrapy.http import HtmlResponse
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

from utils import *


# 爬虫中间件
class JdspiderSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


# 下载中间件
class JdspiderDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        """
        爬虫启动与关闭等信号处理函数
        :param crawler: 爬虫核心引擎
        :return: 传递本方法类
        """
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s

    def __init__(self):
        """
        初始化爬虫程序
        """
        global pic_index
        self.pic_index = 0  # 起始页图片索引
        global time_count
        self.time_count = 0  # 时间计数器

        '''填写登录信息'''
        login_info = True  # 是否开启模拟登录
        self.headless = False  # 是否隐藏浏览器
        self.get_pic = True  # 是否下载图片
        self.sleep = 0.5  # 下拉滚动时延设置
        self.pic_index = 1  # 起始页图片索引
        url = 'https://passport.jd.com/new/login.aspx'  # 设置登录页面
        username = '手机号'  # 填写用账号
        password = '密码'  # 填写密码
        username_id = 'loginname'  # 账号框 id 元素
        password_id = 'nloginpwd'  # 密码框 id 元素
        submit_id = 'loginsubmit'  # 登录按钮 id 元素
        cookies_name = 'jd_cookie.json'  # 生成 cookies 文件名

        '''滑块配置'''
        x, y, b = 2437, 1101, 2.5  # 滑块距离屏幕左上角的 x, y 坐标值, 需根据自己屏幕调整 b 偏置项
        position = {'x': x, 'y': y, 'b': b}
        retry_times = 5  # 设置重试滑块次数

        # 收集登录信息
        info = {'username': username, 'password': password, 'username_id': username_id, 'password_id': password_id,
                'submit_id': submit_id, 'cookies_name': cookies_name, 'position': position, 'retry_times': retry_times}

        # 开始爬取测试
        self.browser = create_chrome_driver(headless=self.headless)
        self.browser.maximize_window()
        self.browser.get(url=url)

        # 自定义代码到达登录框架区域
        if login_info:
            self.browser.find_element_by_class_name('login-tab-r').find_element_by_tag_name('a').click()
            self.browser = login(self.browser, info)
            get_cookies(self.browser, cookies_name)
            add_cookies(self.browser, cookies_name)

    def process_request(self, request: Request, spider):
        """
        自定义请求处理函数
        :param request: 接受蜘蛛程序发送来的请求
        :param spider: 爬虫类
        :return: 返回响应数据
        """
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called

        # 解析动态页面
        print("正在解析页面数据...")
        self.browser.get(request.url)
        # 判断是否为无图模式
        if self.get_pic:
            if request.url.find('Search') != -1:
                spider.logger.info('存在搜索页: %s' % request.url)
                pull_down(self.browser, self.headless, self.sleep)
                try:
                    page_count = request.meta['page_count']
                    WebDriverWait(self.browser, 5, 0.5).until(EC.presence_of_element_located(
                        (By.CSS_SELECTOR,
                         "#J_goodsList > ul > li:nth-child(%d) > div > div.p-name.p-name-type-2 > a > em" % page_count)))
                    spider.logger.info('已找到第 %d 位元素' % page_count)
                    html = etree.HTML(self.browser.page_source)
                    goods_name = request.meta['keyword']
                    download_pic(html, goods_name, self.pic_index, page_count)
                    self.pic_index += page_count
                except Exception as e:
                    spider.logger.error('没有定位到元素: %s' % e)
        else:
            if request.url.find('Search') != -1:
                spider.logger.info('存在搜索页: %s' % request.url)
                self.browser.execute_script('window.scrollTo(0, document.body.scrollHeight)')
                try:
                    page_count = request.meta['page_count']  # 每页最大数量
                    WebDriverWait(self.browser, 5, 0.5).until(EC.presence_of_element_located(
                        (By.CSS_SELECTOR,
                         "#J_goodsList > ul > li:nth-child(%d) > div > div.p-name.p-name-type-2 > a > em" % page_count)))
                    spider.logger.info('已找到第 %d 位元素' % page_count)
                except Exception as e:
                    spider.logger.error('没有定位到元素: %s' % e)
        return HtmlResponse(url=request.url, body=self.browser.page_source,
                            request=request, encoding='utf-8')

    def process_response(self, request: Request, response, spider):
        """
        自定义响应处理函数
        :param request: 接受蜘蛛程序发送来的请求
        :param response: 接受服务器发送来的响应
        :param spider: 爬虫类
        :return: 返回响应数据
        """
        # Called with the response returned from the downloader.
        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest

        # 计算目前总耗时
        self.time_count += 1
        start_time = request.meta['start_time']
        end_time = time.time()
        total_time = end_time - start_time
        m1, s1 = divmod(total_time, 60)
        h1, m1 = divmod(m1, 60)

        # 计算平均总耗时
        average_time = total_time / self.time_count
        m2, s2 = divmod(average_time, 60)
        h2, m2 = divmod(m2, 60)

        # 计算预计总耗时
        predict_time = average_time * len(request.meta['keywords']) * request.meta['page_total']
        m3, s3 = divmod(predict_time, 60)
        h3, m3 = divmod(m3, 60)
        print("-------------------------------")
        print("平均总耗时 => %d时:%02d分:%02d秒" % (h2, m2, s2))
        print("预计总耗时 => %d时:%02d分:%02d秒" % (h3, m3, s3))
        print("-------------------------------")
        print("实际总耗时 => %d时:%02d分:%02d秒" % (h1, m1, s1))
        print("-------------------------------")
        return response

    def process_exception(self, request, exception, spider):
        """
        自定义处理异常函数
        :param request: 接受蜘蛛程序发送来的请求
        :param exception: 接受异常信息
        :param spider: 爬虫类
        """
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        """
        爬虫启动函数执行该函数
        :param spider: 爬虫类
        """
        spider.logger.info('Spider opened: %s' % spider.name)

    def spider_closed(self):
        """
        爬虫关闭函数执行该函数
        """
        self.browser.quit()
        print("爬取完成 => 浏览器驱动已关闭")
